<?php
/**
 *
 * 浣滆� 瑷�厬
 * 閭 xjtdy888[at]163.com
 * QQ 339534039
 * 椤圭洰鎵樼 http://code.google.com/p/html-extractor/
 * 
 */
error_reporting ( E_ALL & ~ E_NOTICE );
class HTMLExtractor
{
	#瑕佸垹鎺夌殑鍏冪礌
	const PC_TAG_DELETE = 1; 
	#瑕佸垹鎺夌殑鏍囩
	const PC_TAG_STRIP = 2;
	static $cleanTags= array(
		array("script",self::PC_TAG_DELETE),
		array("style",self::PC_TAG_DELETE),
		array("link",self::PC_TAG_DELETE),
		array("link",self::PC_TAG_DELETE),
		array("object",self::PC_TAG_DELETE),
		array("embed",self::PC_TAG_DELETE),


		array("p",self::PC_TAG_STRIP),
		array("b",self::PC_TAG_STRIP),
		array("i",self::PC_TAG_STRIP),
		array("u",self::PC_TAG_STRIP),
		array("font",self::PC_TAG_STRIP),
		array("strong",self::PC_TAG_STRIP),
	);
	static $usageTime = 0;
	static function preClean($html)
	{
		foreach(self::$cleanTags as $t)
		{
			if (!$t) continue;
			$name = $t[0];
			$pc   = $t[1];
			$html = preg_replace("#<({$name})(>|\s[^>]*?>)(.*?)</\\1>#is",
				$pc == self::PC_TAG_DELETE ? "" : "\r\n\r\n\\3",$html);				
		}
		return $html;

	}
	static function getDataMainContent($data,$minlength,$maxdepth)
	{
		$s = microtime(true);
		$data = self::preClean($data);
		$root = new htmlTag("document",htmlTag::DOM_TAG);
		$hand = new htmlExtractorHandler($root);
		$p = new htmlParse($data,$hand);
		$p->parse();
		$text =  self::getMainContent($root,$minlength,$maxdepth);
		$e = microtime(true);
		self::$usageTime = $e - $s;
		return $text;
	}
	static function getUrlMainContent($url,$minlength,$maxdepth)
	{
		$data = self::getUrlHtml($url);
		if (!$data) return false;
		return self::getDataMainContent($data,$minlength,$maxdepth);
	}
	static function getDomText($dom,$depth)
	{
		if ($dom->echoset) return ;
		$dom->echoset = true;
		if($dom->depth <= $depth){
			foreach($dom->getChildren() as $child){
				if (is_object($child)){
					$result .= self::getDomText($child,$depth);
				}elseif(is_string($child)){
					$result .= $child;
				}
			}
		}
		return $result;
	}

	static function getMainContent($root,$textLength=100,$maxdepth)
	{

		$result = '';
		$cn  = $root->tagNum + $root->textNum;
		$per = $root->tagNum ? $root->textLength/$textLength / $root->tagNum : 1;
		if ($root->textLength >= $textLength && $per>0.5){
			$result .= self::getDomText($root,$root->depth+$maxdepth);
		}
		foreach($root->getChildren() as $dom){
			if (is_object($dom)){
				$result .= self::getMainContent($dom,$textLength,$maxdepth);
			}
		}
		return $result;
	}

	
	static function checkTextType($url)
	{  
		$url = parse_url($url); 
		if(false != ($fp = @fsockopen($url['host'],empty($url['port'])?80:$url['port'],$error)))
		{
			fputs($fp,"GET ".(empty($url['path'])?'/':$url['path'])." HTTP/1.1\r\n");
			fputs($fp,"Host:$url[host]\r\n\r\n");
			while(!feof($fp))
			{
				$tmp = fgets($fp);
				if(trim($tmp) == ''){
					break;
				}else if(preg_match('#Content-type: text/(.*)#si',$tmp,$arr)){
					fclose($fp);
					return true;
				}
			}
			fclose($fp);
			return false;
		}else{
			return false;
		}
	}

	static function convertToUTF8($str) {
		$charset = mb_detect_encoding($str, array('ASCII','UTF-8','GB2312','GBK','BIG5','ISO-8859-1'));
		if (strcasecmp($charset,'UTF-8') != 0) {
			$str = mb_convert_encoding($str,'UTF-8',$charset);
		}
		return $str;
	}


	static function getUrlHtml($url){
		//return file_get_contents("txt.txt");
		if (!self::checkTextType($url)){
			exit();
		}
		return file_get_contents($url);
	}
}
/**
 *
 * HTML 鏍囩瑙ｆ瀽鍣�
 * 璇ヨВ鏋愬櫒浠�>涓哄崟鍏�姣斿 <div id="cc"> 杩欐槸涓�釜澶勭悊鍗曞厓
 * 鎵�互 <div></div> 杩欏彞鏄�涓鐞嗗崟鍏�div>鍜�/div>
 * 瑙ｆ瀽鍣ㄦ瘡澶勭悊涓�釜鍗曞厓閮戒細浜х敓鍥炶皟鍑芥暟锛岃嚦浜庢�涔堟潵澶勭悊杩欎釜鍗曞厓鐢卞鐞嗗櫒鏉ュ喅瀹�
 * 涔熷氨鏄璇ヨВ鏋愬櫒骞朵笉鍘诲鐞嗘爣绛惧尮涓嶅尮閰嶄箣绫荤殑闂
 * </span></span> 杩欐牱鐨勫瓧绗︿覆涔熸槸鍙互杩涜瑙ｆ瀽鐨勶紝浜х敓2娆�endElement 浜嬩欢鍥炶皟銆�
 * 鏈潵鍏堢敤PHP鑷甫鐨凞OM瀵硅薄绫伙紝涓嶈繃鐢变簬澶ч儴浠界綉椤甸兘涓嶈鑼冿紝瑙ｆ瀽璧锋潵澶ч儴浠芥槸浼氬嚭閿欑殑
 * 鎵�互鑷繁鍐欎簡杩欎釜绠�崟鐨�
 * 鏈被鎬诲叡3涓洖璋冨嚱鏁�
 *  startElement($parser,$tagName)	鍙戠幇寮�鏍囩
 *  endElement($parser,$tagName)	鍙戠幇闂悎鏍囩
 *  characterData($parser,$char)	鍙戠幇鏍囩鍐呭
 *  鏈被娌″仛浠讳綍浼樺寲锛屾墍浠ュ洖璋冪殑棰戠巼浼氱浉褰撶殑楂樸�
 *
 */
class htmlParse
{
	/** 
	 *	瑕佸鐞嗙殑HTML鍐呭
	 */
	protected $_html = '';
	/**
	 * _html 鐨勯暱搴�
	 */
	protected $_htmlLength = 0;
	/** 
	 * 褰撳墠澶勭悊浣嶇疆鎸囬拡
	 */
	protected $_pt = 0;
	/**
	 * 鏍囩鐘舵�鏍�
	 */
	protected $_tagStatus = array();
	/**
	 * 鏍囩鏍�
	 */
	protected $_tagStack = array();
	/**
	 * 褰撳墠鏍囩鍚嶇О
	 */
	protected $_tagName = '';
	
	/** 
	 * 鏍囩寮�鏍囪瘑
	 */
	const TAG_START = 10;
	/**
	 * 鏍囩缁撴潫鏍囪瘑
	 */
	const TAG_END   = 20;

	/**
	 * 鏍囩鍚嶅瓧寮�
	 */
	const TAGNAME_START = 30;
	/**
	 * 鏍囩鍚嶅瓧缁撴潫
	 */
	const TAGNAME_END = 40;
	/**
	 * 娉ㄩ噴寮�(淇濈暀锛�
	 */
	const COMMENT_START = 50;
	/**
	 * 娉ㄩ噴缁撴潫(淇濈暀锛�
	 */
	const COMMENT_END = 60;


	/** 
	 *  浜嬩欢鍥炶皟瀵硅薄
	 */
	public $_elementHandler = null;
	/**
	 * 
	 * 鏋勫嚱鏂规硶
	 * @param striing $html 瑕佽В鏋愮殑瀛楃涓�
	 * @param object|array elementHandler 鍥炶皟澶勭悊鍣ㄥ彲浠ユ槸鏁扮粍涔熷彲浠ユ槸瀵硅薄
	 *		瀵硅薄鍙瀹炵幇鐩稿悓鐨勬柟娉曞悕灏卞彲浠ワ紝娉ㄦ剰杩欓噷娌℃湁鐢ㄥ埌鎺ュ彛
			濡傛灉鏄暟缁勶紝鏂规硶鍚嶄綔涓轰笅鏍囧嵆鍙�
	 *
	 */
	public function htmlParse($html,$elementHander=null)
	{
		$this->setHtml($html);
		$this->setElementHandler($elementHander);
	}
	/**
	 *
	 * 閲嶆柊璁惧畾瑕佽В鏋愮殑鍐呭
	 * @param string $html
	 *
	 */
	public function setHtml($html)
	{
		$this->_html = $html;
		$this->_reset();
	}
	/**
	 *
	 * 閲嶄綅澶勭悊鎸囬拡锛岃澶勭悊鐨勫瓧绗﹂暱搴�
	 *
	 */
	public function _reset()
	{
		$this->_pt = 0;
		$this->_htmlLength = strlen($this->_html);
	}
	/**
	 * 閲嶆柊鎸囧畾澶勭悊鍣�
	 * @param object|array elementHandler
	 */
	public function setElementHandler($elementHander)
	{
		$this->_elementHandler = $elementHander;
	}
	/**
	 * 鑾峰彇瑕佸鐞嗙殑涓嬩竴涓瓧绗�鎸囬拡鑷姩鍚庣Щ
	 * 鍒扮粨灏句簡杩斿洖false
	 * @return char
	 */
	public function nextChar()
	{
		if ($this->_pt < $this->_htmlLength){
			return $this->_html[$this->_pt++];
		}
		
		return false;
	}
	/**
	 * 鑾峰彇澶勭悊杩囩殑涓婁竴涓瓧绗︽寚閽堝洖閫�
	 * 鍒扮粨灏句簡杩斿洖false
	 * @return char
	 */
	public function preChar()
	{
		if ($this->_pt > 0){
			return $this->_html[--$this->_pt];
		}
		return false;
	}
	/**
	 * 鑾峰緱褰撳墠澶勭悊浣嶇疆
	 * @return integer
	 */
	public function getPt()
	{
		return $this->_pt;
	}
	/**
	 * 璁剧疆澶勭悊浣嶇疆 鎴愬姛杩斿洖true 澶辫触false
	 * @return bool
	 */
	public function setPt($v)
	{
		if ($v>-1 && $v < $this->_htmlLength){
			$this->_pt = $v;
			return true;
		}
		return false;
	}

	public function addTagStack()
	{
		return array_push($this->_tagStack,$this->_tagName);
	}

	public function startElement($parse,$tagName)
	{
	}

	public function endElement($parse,$tagName)
	{
	}
	public function characterData($parse,$char)
	{
	}

	public function endParse($parse)
	{
	}

	public function callHandler($callback)
	{
		$argv = func_get_args();
		array_shift($argv);
		array_unshift($argv,$this);
		if (is_array($this->_elementHandler) && $this->_elementHandler[$callback]){
			return call_user_func($this->_elementHandler[$callback],$argv);
		}else{
			$handler = is_object($this->_elementHandler) ? $this->_elementHandler : $this;
			if (method_exists($handler,$callback)){
				return call_user_func_array(array($handler,$callback),$argv);
			}
		}
	}

	public function parse()
	{
		while(($char=$this->nextChar()) !== false)
		{
			switch($char)
			{
				case '<':
					if (!$this->_tagStatus || end($this->_tagStatus) == self::TAG_END)	
					{
						$pt = $this->getPt();

						$char1 = $this->nextChar();
						$char2 = $this->nextChar();
						$char3 = $this->nextChar();
						$refor = false;
						if ($char1 == '!' && ($char2 == '-' && $char3 == '-')) {
							//濡傛灉鏄敞閲�
							while(($char1=$this->nextChar()) !== false)
							{
								
								if ($char1 != '>') continue;
								$pt2 = $this->getPt();
								$this->preChar();
								$char2 = $this->preChar();
								$char3 = $this->preChar();
								if ($char2 == '-' && $char3 == '-') {

									$refor = true;
									$this->setPt($pt2);
									break;
								}
								$this->setPt($pt2);
							}
							
						}
						if ($refor){
							continue;
						}

						$this->setPt($pt);
	
						array_push($this->_tagStatus,self::TAG_START);
						array_push($this->_tagStatus,self::TAGNAME_START);
						$this->_tagName = '';
					}
				break;
				case ' ':  case '>':
					if (!$this->_tagStatus || end($this->_tagStatus) == self::TAG_END){
						$callback = 'characterData';
						$this->callHandler($callback,$char);
						continue;
					}
					$callback = '';
					if (end($this->_tagStatus) == self::TAGNAME_START) {
						array_pop($this->_tagStatus);
						array_push($this->_tagStatus , self::TAGNAME_END);
						if ($this->_tagName[0] == '/'){
							$this->_tagName = substr($this->_tagName,1);
							$callback = 'endElement';
						}else{
							$callback = 'startElement';
						}
					}
					// <p /> <p/> <p / >
					// <link ... />
					if (in_array(end($this->_tagStatus) ,array(self::TAGNAME_START,	self::TAGNAME_END)) && $char == '>'){
						$pt = $this->getPt();
						$this->setPt($pt-1);
						while(($char2=$this->preChar()) !== false && !preg_match("#\s#",$char2)){
							if ($char2 == '/'){
								//鑷棴鍚堟爣绛�
								$callback = 'endElement';
								array_pop($this->_tagStatus); //end tagname_start
								array_push($this->_tagStatus , self::TAGNAME_END);
							}
							break;
						}
						$this->setPt($pt);
					}
					if ($callback == 'startElement'){
						$this->addTagStack();
						$this->callHandler($callback,$this->_tagName);
					}elseif ($callback == 'endElement'){

						array_pop($this->_tagStatus); //end tagname
						array_pop($this->_tagStatus); // end tag
						$this->callHandler($callback,$this->_tagName);
					}
					if (end($this->_tagStatus) == self::TAGNAME_END && $char == '>'){
						array_pop($this->_tagStatus); //end tag name
						array_pop($this->_tagStatus); //end tag
					}
				break;
				default:
					if (end($this->_tagStatus) == self::TAGNAME_START)
					{
						$this->_tagName .= $char;
					}
					if (!$this->_tagStatus || end($this->_tagStatus) == self::TAG_END){
						$callback = 'characterData';
						$this->callHandler($callback,$char);
					}
				break;
			}
		}
		$callback = 'endParse';
		$this->callHandler($callback,$char);
	}
}
/**
 */
class htmlTag
{
	public $tagName = '';
	public $type = '';
	public $depth = 0;
	public $parent = null;
	public $childs = array();

	public $textLength = 0;

	public $tagNum  = 0;
	public $textNum = 0;

	const DOM_TAG = 1;

	public function __construct($tagName,$type)
	{
		$this->type = $type;
		$this->tagName = $tagName;
	}

	public function addChild($child)
	{
		array_push($this->childs,$child);
		if (!is_object($child)){
			$this->textLength += $this->_strlen($child,true);
			$this->textNum++;
		}else{
			$this->tagNum++;
		}
	}
	public function _strlen($text,$ignoreSpace=false)
	{
		if ($ignoreSpace) $text = preg_replace("#\s*#s","",$text);
		return strlen($text);
	}

	public function getChildren()
	{
		$result = array();
		foreach($this->childs as $dom)
		{
			$result[] = $dom;
		}
		return $result;
		
	}
	public function getText()
	{
		$text = '';
		foreach($this->childs as $dom){
			if (is_string($dom)) $text .= $dom;
		}
		return $text;
	}
}

class htmlExtractorHandler
{
	public $ignoreTags=array(
		"!doctype","meta","link","hr","!--","base","basefont","br",
		"frame","frameset","noframes","iframe",
		"input","button","select","optgroup","option",
		"label","fieldset","legend","isindex",
		"img","map","area","style",
		"script","noscript","applet","object","param","marquee","embed");

	protected $_dom = array();

	private $_charBuffer = '';
	private $_domDepth   = 0;

	public function __construct($root)
	{
		array_push($this->_dom,$root);
	}

	public function isIgnore($tag)
	{
		$tag = strtolower($tag);
		return in_array($tag,$this->ignoreTags);
	}

	public function endParse()
	{
		$this->updateChacter();
	}

	public function updateChacter()
	{
		if ($this->_charBuffer != ''){
			end($this->_dom)->addChild($this->_charBuffer);
			$this->_charBuffer = '';
		}
	}

	public function startElement($parse,$tagName)
	{
		$this->updateChacter();
		$tagName = strtolower($tagName);
		if ($this->isIgnore($tagName) === true) return false;

		$dom = new htmlTag($tagName,htmlTag::DOM_TAG);
		$parent = end($this->_dom);
		$dom->parent = $parent;
		//echo str_repeat("    ",$this->_domDepth)."[{$dom->tagName}_{$this->_domDepth}]\r\n";
		$dom->depth  = ++$this->_domDepth;
		$parent->addChild($dom);
		

		array_push($this->_dom,$dom);
	}

	public function endElement($parse,$tagName)
	{
		$this->updateChacter();
		$tagName = strtolower($tagName);
		if ($this->isIgnore($tagName) === true) return false;
		$dom = end($this->_dom);
		if (end($this->_dom)->tagName == $tagName) {
			array_pop($this->_dom);
			$this->_domDepth--;
		}
		//echo str_repeat("    ",$this->_domDepth)."[/{$dom->tagName}_{$this->_domDepth}]\r\n";
	}
	public function characterData($parse,$char)
	{
		$this->_charBuffer .= $char;
	}
}
